#!/usr/bin/env -S deno run --allow-net --allow-write --allow-read
import { DOMParser, Element } from "https://deno.land/x/deno_dom/deno-dom-wasm.ts";
import { render } from 'https://deno.land/x/mustache_ts/mustache.ts';
import { dirname } from "https://deno.land/std/path/mod.ts";
import { writeCSV as writeCSVFile } from "https://deno.land/x/csv/mod.ts";
import {stringify as yamlStringify } from 'https://deno.land/std@0.82.0/encoding/yaml.ts';
import { delay } from "https://deno.land/std@0.167.0/async/delay.ts";
import * as log from "https://deno.land/std/log/mod.ts";


// 2021 数据入口
const PROVINCE_URL = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/index.html"

interface Output{
    /**
     * 文件名，包含路径。比如 java/foo/bar/Data.java
     */
    name: string
    /**
     * 文件模板
     */
    template: string
}

interface Region{
    name: string
    // 补齐位数？
    code: string
    parentCode?: string
    url?: string
}


// 保存已抓取的url
const JSON_TEMPLATE = "@json"
const YAML_TEMPLATE = "@yaml"
const CSV_TEMPLATE = "@csv"
const CRAWLED_URLS: Set<string> = new Set()
const OUTPUTS: Output[] = [
    {
        name: "java/chineseregion/Region.java",
        template: `
package chineseregion;
    public abstract class Region{
    public static String[][] DATA = {
{{#regions}}       {"{{code}}","{{name}}"},\n{{/regions}}
    };
}`
    },
    {
        name: "ruby/region.rb",
        template: `
module Chineseregion
    DATA = [
{{#regions}}       ["{{code}}","{{name}}"],\n{{/regions}}
    ]
end`
    },
    {
        name: "go/region/region.go",
        template: `
package region

var Data = [][]string{
{{#regions}}    {"{{code}}","{{name}}"},\n{{/regions}}
}`
    },
    {
        name: "php/region.php",
        template: `<?php namespace chineseregion;
    const DATA = [
{{#regions}}           ["{{code}}","{{name}}"],\n{{/regions}}
    ];
?>`
    },
    {
        name: "sql/region.sql",
        template: `DROP TABLE IF EXISTS chinese_regions;

CREATE TABLE chinese_regions(
    id bigint NOT NULL AUTO_INCREMENT,
    code VARCHAR(12) NOT NULL COMMENT '代码',
    name VARCHAR(64) NOT NULL COMMENT '名称',
    PRIMARY KEY (id),
    UNIQUE KEY code_idx (code),
    KEY name_idx (name)
) COMMENT='中国区域码表';

{{#regions}}INSERT INTO chinese_regions (code, name) VALUES('{{code}}', '{{name}}');\n{{/regions}}`
    },
    {
        name: "csv/region.csv",
        template: CSV_TEMPLATE
    },
    {
        name: "json/region.json",
        template: JSON_TEMPLATE
    },
    {
        name: "yaml/region.yaml",
        template: YAML_TEMPLATE
    }
]

/**
 * 从链接获取编码 
 * 
 * http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/11.html => 11 
 * 
 * 11.html => 11
 * @param url 详情页地址
 */
function parseCodeFromHref(url: string): string{
    const pattern = /\/*\/?(\d+)\.html$/;
    const match = url.match(pattern)
    if(match){
        return match[1]
    }else{
        throw new Error("无效url " + url)
    }
}

/**
 * 把链接转换为url
 * @param href 链接地址。比如 a.html 或者 http://www.exmaple.org
 * @param pageUrl 链接所在页面的绝对地址，比如 http://www.exmaple.org
 * @returns 绝对地址 比如 http://www.exmaple.org/a.html
 */
function toUrl(href: string, pageUrl: string): string{
    if(!pageUrl.startsWith("http")){
        throw new Error("页面地址需要http(s)开头")
    }
    if(href.startsWith("http")){
        return href
    }else{
        return new URL(href, pageUrl).href
    }
}

function parseProvince(html: string, url: string): Region[]{
    const document = new DOMParser().parseFromString(html, "text/html")!
    const links = document.querySelectorAll(".provincetable a")
    const regions: Region[] = []
    links.forEach(link => {
        const href = (link as Element).getAttribute("href")!
        regions.push({
            code: parseCodeFromHref(href).trim().padEnd(12, "0"),
            name: (link as Element).innerText.trim(),
            url: toUrl(href, url)
        })
    })
    return regions
}


function parseCity(html: string, url: string): Region[]{
    const document = new DOMParser().parseFromString(html, "text/html")!
    const trs = document.querySelectorAll(".citytr")
    const regions: Region[] = []
    trs.forEach(tr => {
        const tds = (tr as Element).querySelectorAll("td")
        const a = (tr as Element).querySelector("a") as Element
        const pageUrl = a && toUrl(a.getAttribute("href")!, url)
        regions.push({
            code: (tds[0] as Element).innerText.trim().padEnd(12, "0"),
            name: (tds[1] as Element).innerText.trim(),
            url: pageUrl
        })
    })
    return regions
}

function parseStreet(html: string, url: string): Region[]{
    const document = new DOMParser().parseFromString(html, "text/html")!
    const trs = document.querySelectorAll(".towntr")!
    const regions: Region[] = []
    trs.forEach(tr => {
        const tds = (tr as Element).querySelectorAll("td")
        const a = (tr as Element).querySelector("a") as Element
        const pageUrl = a && toUrl(a.getAttribute("href")!, url)
        regions.push({
            code: (tds[0] as Element).innerText.trim().padEnd(12, "0"),
            name: (tds[1] as Element).innerText.trim(),
            url: pageUrl
        })
    })
    return regions
}

function parseCommunity(html: string, _url: string): Region[]{
    const document = new DOMParser().parseFromString(html, "text/html")!
    const trs = document.querySelectorAll(".villagetr")
    const regions: Region[] = []
    trs.forEach(tr => {
        const tds = (tr as Element).querySelectorAll("td")
        regions.push({
            code: (tds[0] as Element).innerText.trim().padEnd(12, "0"),
            name: (tds[2] as Element).innerText.trim()
        })
    })
    return regions
}


function parseDistrict(html: string, url: string): Region[]{
    const document = new DOMParser().parseFromString(html, "text/html")!
    const trs = document.querySelectorAll(".countytr")
    const regions: Region[] = []
    trs.forEach(tr => {
        const tds = (tr as Element).querySelectorAll("td")
        const a = (tr as Element).querySelector("a")! as Element
        const pageUrl = a && toUrl(a.getAttribute("href")!, url)
        regions.push({
            code: (tds[0]as Element).innerText.trim().padEnd(12, "0"),
            name: (tds[1] as Element).innerText.trim(),
            url: pageUrl
        })
    })
    return regions
}

// 延迟 1s，抓取太快会301
const DELAY_PER_URL = 1

// 如果被限流，等1小时
const WAIT_SECS = 3600

// 限流标识
let blocked = false

async function getUrl(url: string): Promise<string>{
    log.info("开始处理 " + url)
    const delaySecs = blocked ? WAIT_SECS : DELAY_PER_URL
    if(delaySecs > 1){
        log.info("延迟 ", delaySecs, '秒')
    }
    await delay(1000 * delaySecs)
    return fetch(url)
    .then(r =>{
        blocked = false
        return r.text()
    })
    .catch(e => {
        log.error("请求错误", url, e)
        blocked = true
        // TODO 不要用递归
        return getUrl(url)
    })
    
}
    

async function crawlAndParse(urls: string[], parser: (html: string, url: string) => Region[]): Promise<Region[][]>{
    const result: Region[][] = []
    // 串行处理，并行会301
    for(const url of urls){
        if(CRAWLED_URLS.has(url)){
            continue
        }
        const text = await getUrl(url)
        CRAWLED_URLS.add(url)
        result.push(parser(text, url))
    }
    return Promise.resolve(result);
}

async function makeDirIfNeeded(dir: string){
    try {
        await Deno.mkdir(dir, { recursive: true });
    } catch (error) {
        if (!(error instanceof Deno.errors.AlreadyExists)) {
            throw error;
        }
    }
}

function writeJson(fileName: string, regions: Region[]): Promise<void>{
    return Deno.writeTextFile(fileName, JSON.stringify(regions.map(r => [r.code, r.name])));
}

async function writeCsv(fileName: string, regions: Region[]): Promise<void>{
    const f = await Deno.open(fileName, {
        write: true,
        create: true,
        truncate: true,
    });
    const rows = regions.map(r => [r.code, r.name])
    await writeCSVFile(f, rows);
    f.close();
}

async function writeYaml(fileName: string, regions: Region[]){
    return Deno.writeTextFile(fileName, yamlStringify(regions.map(r => [r.code, r.name])));
}

async function writeMustache(fileName: string, template: string, regions: Region[]){
    const content = render(template, {regions})
    return Deno.writeTextFile(fileName, content);
}


function writeOutput(regions: Region[], outputs: Output[]): Promise<void[]>{
    const jobs = outputs.map(async (output) => {
        const dir = dirname(output.name)
        await makeDirIfNeeded(dir)
        switch(output.template){
            case YAML_TEMPLATE: return writeYaml(output.name, regions)
            case JSON_TEMPLATE: return writeJson(output.name, regions)
            case CSV_TEMPLATE: return writeCsv(output.name, regions)
            default: return writeMustache(output.name, output.template, regions)
        }
    })
    return Promise.all(jobs);
}

async function main(){
    let provinces = (await crawlAndParse([PROVINCE_URL], parseProvince)).flat()
    let cities = (await crawlAndParse(provinces.filter(p => p.url).map(p => p.url!), parseCity)).flat()
    let districts = (await crawlAndParse(cities.filter(p => p.url).map(c => c.url!), parseDistrict)).flat()
    let streets = (await crawlAndParse(districts.filter(p => p.url).map(c => c.url!), parseStreet)).flat()
    let communities = (await crawlAndParse(streets.filter(p => p.url).map(c => c.url!), parseCommunity)).flat()
    // const regions = await (await crawlAndParse(["http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/12/01/02/120102003.html"], parseCommunity)).flat()
    await writeOutput([...provinces, ...cities, ...districts, ...streets, ...communities], OUTPUTS)
}


main();